import pandas as pd
import os
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import time

# 读取 Excel 文件中的企业名称列表
input_file = 'F:\\qcc3.xlsx'
output_file = 'F:\\企查查数据.xlsx'
# 读取 Excel 文件
df = pd.read_excel(input_file)
urls = df['url'].tolist()

# 初始化结果列表
results = []

# 设置 Chrome 选项
chrome_options = Options()

# 添加基本的请求头信息
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
# 初始化 WebDriver
service = Service(executable_path=r'F:\chromedriver-win64\chromedriver.exe')
driver = webdriver.Chrome(service=service, options=chrome_options)

results = []

#
# # 设置复杂的请求头信息
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Encoding": "gzip, deflate, br, zstd",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "www.cmde.org.cn",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "sec-ch-ua": "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\""
}

# 使用 CDP 命令设置请求头
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
            Object.defineProperty(navigator, 'webdriver', {
              get: () => undefined
            })
          """
})

# 创建下载目录
download_dir = r'F:\knowledge'
if not os.path.exists(download_dir):
    os.makedirs(download_dir)

# 爬取数据
for url in urls:
    driver.get(url)
    try:
        # 等待元素加载
        wait = WebDriverWait(driver, 15)  # 最多等待 15 秒
        result_div = wait.until(EC.presence_of_element_located((By.XPATH, "//div[@class='wenzhang w1200-auto']")))
        # 查找第一个搜索结果
        if result_div:
            title_element = result_div.find_element(By.CSS_SELECTOR, 'h2.title')
            title_tag=title_element.text.strip() if title_element else ''
            two_title_element = result_div.find_element(By.CSS_SELECTOR, 'h3.two-title')
            two_title_tag = two_title_element.text.strip() if two_title_element else ''
            # 检查 content_tag 是否存在
            content_tag_element = result_div.find_element(By.CSS_SELECTOR, 'div.text')
            content_tag = content_tag_element.text.strip() if content_tag_element else ''
            date_tag_element = result_div.find_element(By.CSS_SELECTOR, 'div.date')
            date_tag=date_tag_element.text.strip() if date_tag_element else ''
            # 检查附件是否存在
            try:
                # 使用 JavaScript 过滤掉具有 name 属性的 <a> 标签
                attach_tags = driver.execute_script("""
                    var links = arguments[0].querySelectorAll('div.text > p > a');
                    var filteredLinks = Array.from(links).filter(function(link) {
                        return !link.hasAttribute('name');
                    });
                    return filteredLinks;
                """, result_div)
                #attach_tags = result_div.find_elements(By.CSS_SELECTOR, 'div.text > p > a')
                attach_names = []
                attach_links = []
                for attach_tag in attach_tags:
                    attach_name = attach_tag.text.strip() if attach_tag else ''
                    attach_link = attach_tag.get_attribute('href') if attach_tag else ''
                    attach_names.append(attach_name)
                    attach_links.append(attach_link)

                # 将附件名称和链接以逗号分隔的形式存储
                attach_names_str = ','.join(attach_names)
                attach_links_str = ','.join(attach_links)
                results.append({
                    '地址': url,
                    '标题': title_tag,
                    '副标题': two_title_tag,
                    '内容': content_tag,
                    '发布日期': date_tag,
                    '附件名称': attach_names_str,
                    '附件链接': attach_links_str
                })
            except NoSuchElementException:
                results.append({
                    '地址': url,
                    '标题': title_tag,
                    '副标题': two_title_tag,
                    '内容': content_tag,
                    '发布日期': date_tag,
                    '附件名称': None,
                    '附件链接': None
                })
        else:
            results.append({'地址': url, '备注': '未找到相关信息'})
    except TimeoutException:
        results.append({'地址': url, '备注': '请求超时'})
    except Exception as e:
        results.append({'地址': url, '备注': f'请求失败，错误信息: {str(e)}'})

    # 设置请求间隔，避免被封禁
    time.sleep(1)

# 关闭 WebDriver
driver.quit()

# 将结果写入新的 Excel 文件
result_df = pd.DataFrame(results)
result_df.to_excel(output_file, index=False)

print(f'数据已保存到 {output_file}')
